package com.changxianggu.crawler.cloud.test;

import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import com.changxianggu.cloud.crawler.core.WebApplication;
import com.changxianggu.cloud.crawler.core.modules.storage.entity.CxgBook;
import com.changxianggu.cloud.crawler.core.modules.storage.service.ICxgBookService;
import com.changxianggu.cloud.crawler.core.utils.HttpUtil;
import com.changxianggu.cloud.crawler.core.utils.UserAgentUtil;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = {WebApplication.class})// 指定启动类
@Slf4j
public class WaiJiaoShe {

    @Autowired
    ICxgBookService bookService;

    public static final String COOKIE = "Hm_lvt_7ee4d11fe53d77f37db1e9f200336484=1582255150; _ga=GA1.2.432766063.1583216012; ASP.NET_SessionId=eccc1ixvy1ocznykxe0bobwt; _gid=GA1.2.2082765804.1587541870; _gat=1";
    public static final String REFERER = "https://we.sflep.com/books/books.aspx?kw=";
    public static final String URL_PRE = "https://we.sflep.com/books/";


    @Test
    public void initCrawler() throws Exception {
        doCrawler();
    }

    public void doCrawler() throws Exception {
        int pageNum =22;
        int totalPage = 27;
        for (int i = pageNum; i <= totalPage; i++) {
            String url = "https://we.sflep.com/books/books.aspx";
            Map<String, Object> parameter = MapUtil.newHashMap(2);
            parameter.put("kw", "");
            parameter.put("k", 1);
            parameter.put("p", i);
            Map<String, Object> headers = MapUtil.newHashMap(5);
            headers.put("Cookie", COOKIE);
            headers.put("User-Agent", UserAgentUtil.random());
            headers.put("Referer", REFERER);
            String result = HttpUtil.doGet(url, headers,parameter);
            while (result.contains("页面没有找到")) {
                //睡眠2秒继续访问
                Thread.sleep(2000);
                result = HttpUtil.doPost(url, headers);
            }
            if (StrUtil.isNotBlank(result)) {
                Document htmlDoc = Jsoup.parse(result);
                Elements bookContentList = htmlDoc.select(".resource_list");
                if (bookContentList != null && bookContentList.size() > 0) {
                    List<CxgBook> listBook = new ArrayList<>();
                    for (Element element : bookContentList) {
                        Element detailElement = element.select(".media-left a").first();
                        if (detailElement != null) {
                            String detailUri = detailElement.attr("href");
                            CxgBook bookEntity = getBookEntity(URL_PRE+detailUri);
                            if (StrUtil.isNotBlank(bookEntity.getBookIsbn())) {
                                listBook.add(bookEntity);
                            }
                        }
                    }
                    bookService.saveBatch(listBook);
                    log.info("第" + i + "页数据保存成功");
                    log.warn("--->" + i);
                }
            }
        }
    }

    public CxgBook getBookEntity(String uri) throws Exception {
        CxgBook bookEntity = new CxgBook();
        Map<String, Object> headers = MapUtil.newHashMap(5);
        headers.put("Cookie", COOKIE);
        String result = HttpUtil.doGet(uri, headers);
        while (result.contains("页面没有找到")) {
            //睡眠2秒继续访问
            Thread.sleep(2000);
            result = HttpUtil.doPost(uri, headers);
        }
        if (StrUtil.isNotBlank(result)) {
            Document htmlDoc = Jsoup.parse(result);
            Element bookContent = htmlDoc.select("body").first();
            if (bookContent != null) {
                bookEntity.setPkId(getNumbers(uri));

                Element isbnElement = bookContent.select(".book_list_disc ul").eq(1).first();
                if (isbnElement != null) {
                    String isbn = isbnElement.text().trim().replace("ISBN：","").replace("-","");
                    String[] isbnArr = isbn.split("，");
                    bookEntity.setBookIsbn(isbnArr[0]);
                }

                Element nameElement = bookContent.select(".media-heading").first();
                if (nameElement != null) {
                    String name = nameElement.text();
                    bookEntity.setBookName(name);
                }

                Element authorElement = bookContent.select(".book_list_disc ul").first();
                if (authorElement != null) {
                    String author = authorElement.text().replace("作者：","");
                    bookEntity.setAuthor(author);
                }

                Element coverElement = bookContent.select(".media-object").first();
                if (coverElement != null) {
                    String coverStr = coverElement.attr("src");
                    if (StrUtil.isNotBlank(coverStr)) {
                        bookEntity.setCover("https://we.sflep.com"+coverStr);
                    }
                }

                Element cataElement = bookContent.select(".spic_but button").first();
                if (cataElement != null) {
                    String cataStr = cataElement.attr("onclick");
                    cataStr = subString(cataStr,"open('","')");
                    if (StrUtil.isNotBlank(cataStr)) {
                        bookEntity.setCatalog(cataStr);
                    }
                }

                Element digUriElement = bookContent.select("#book_pdf").first();
                if (digUriElement != null) {
                    String digUriStr = digUriElement.attr("onclick");
                    digUriStr = subString(digUriStr,"open('","?title");
                    if (StrUtil.isNotBlank(digUriStr)) {
                        bookEntity.setDigitalUri(digUriStr);
                    }
                }

                bookEntity.setPressName("上海外语教育出版社");
                bookEntity.setStatus(1);
                bookEntity.setPressId(46);
                bookEntity.setIsSync(0);
                bookEntity.setCreateTime(LocalDateTime.now());

                //brief
                Element briefElement = bookContent.select(".content_txt").first();
                if (briefElement != null) {
                    String briefStr = briefElement.text();
                    if (StrUtil.isNotBlank(briefStr)) {
                        bookEntity.setBrief(briefStr);
                    }
                }

                //pub_date
                Element pubDateElement = bookContent.select(".book_list_disc ul").eq(2).first();
                if (pubDateElement != null) {
                    String pubDateStr = pubDateElement.text().replace("出版日期：","").trim();
                    if (StrUtil.isNotBlank(pubDateStr)) {
                        try {
                            if (pubDateStr.contains("年")) {
                                pubDateStr = pubDateStr.replace("年","-").replace("月","-");
                            }else {
                                String[] dateArr = pubDateStr.split("-");
                                if (dateArr.length>2) {
                                    pubDateStr = pubDateStr + " 00:00:00";
                                }else {
                                    pubDateStr = pubDateStr + "-01 00:00:00";
                                }
                            }
                            DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
                            LocalDateTime ldt = LocalDateTime.parse(pubDateStr,df);
                            bookEntity.setPubDate(ldt);
                        }catch (Exception e){

                        }
                    }
                }

            }
        }
        return bookEntity;
    }

    /**
     * 判断一个字符串是否都为数字
     *
     * @param strNum
     * @return
     */
    public boolean isDigit(String strNum) {
        Pattern pattern = Pattern.compile("[0-9]{1,}");
        Matcher matcher = pattern.matcher((CharSequence) strNum);
        return matcher.matches();
    }

    /**
     * 截取数字
     *
     * @param content
     * @return
     */
    public String getNumbers(String content) {
        Pattern pattern = Pattern.compile("\\d+");
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return "";
    }

    /**
     * 截取字符串str中指定字符 strStart、strEnd之间的字符串
     * @return
     */
    public String subString(String str, String strStart, String strEnd) {

        /* 找出指定的2个字符在 该字符串里面的 位置 */
        int strStartIndex = str.indexOf(strStart);
        int strEndIndex = str.indexOf(strEnd);

        /* index 为负数 即表示该字符串中 没有该字符 */
        if (strStartIndex < 0) {
            return "";
        }
        if (strEndIndex < 0) {
            return "";
        }
        /* 开始截取 */
        String result = str.substring(strStartIndex, strEndIndex).substring(strStart.length());
        return result;
    }
}
